// crawl-book.js
// 知源中医 - 按“卷”合并内容，每卷一个 Markdown 文件
const puppeteer = require('puppeteer');
const fs = require('fs');
const path = require('path');
const TurndownService = require('turndown');
const { sleep, deleteFolderRecursive } = require('./common/util');

const baseSite = '知源中医';
const outputDir = './docs/' + baseSite;
const allowBooks = '6,7,8,1,5,26,535,394,851,1419,813,295,289,1443,35,1440,1426,73,1438,113,1424,127,1423,1446,1444,1441,1430,365,493,478,1418,320,487,178,156,167,166,145,164,163,355,9,447,449,446,407,458,367,420,475,534,531,437,372,223,228,797,1436,267,260,544,503,381,520,517,818,925,529,366,547,1344,189,772,747,775,1007,559,491,316,328,411'.split(',').map(Number)
let browser = null;
let browserPid = null;
if (!fs.existsSync(outputDir)) fs.mkdirSync(outputDir, { recursive: true });

const turndownService = new TurndownService({
  headingStyle: 'atx',
  codeBlockStyle: 'fenced',
  emDelimiter: '*',
});

// 增强转换规则
turndownService.addRule('images', {
  filter: ['img'],
  replacement: function (content, node) {
    const src = node.getAttribute('src');
    const alt = node.getAttribute('alt') || '';
    return `[${alt}](${src})`;
  }
});

turndownService.addRule('blockquotes', {
  filter: ['blockquote'],
  replacement: function (content) {
    return `\n> ${content.trim().replace(/\n/g, '\n> ')}\n\n`;
  }
});

/**
 * 检查一本书是否所有卷都已完整爬取（无失败、无遗漏）
 * @param {string} bookDir - 书籍本地存储目录
 * @param {Object} book - 书籍元信息 { id, name }
 * @param {Array} volumes - bookData.volumes 数组
 * @returns {boolean} 是否全部完成
 */
function isBookComplete(bookDir, book, volumes) {
  const failedKeywords = ['爬取失败：', '内容未获取到。', '本卷为空，无内容。'];
  for (const vol of volumes) {
    // 生成安全的文件名
    const safeVolumeTitle = vol.volumeTitle.replace(/[\\/:\*\?\"<>\|]/g, '_');
    const volumeFilename = `${String(vol.index + 1).padStart(3, '0')}_${safeVolumeTitle}.md`;
    const volumeFilePath = path.join(bookDir, volumeFilename);

    // ✅ 检查文件是否存在
    if (!fs.existsSync(volumeFilePath)) {
      console.log(`🔍 未找到文件: ${volumeFilename}`);
      return false;
    }

    // ✅ 检查文件内容是否包含“爬取失败”
    try {
      const content = fs.readFileSync(volumeFilePath, 'utf8');
      if (failedKeywords.some(keyword => content.includes(keyword))) {
        console.log(`🔍 文件中包含失败标记: ${volumeFilename}`);
        return false;
      }
    } catch (err) {
      console.error(`❌ 读取文件失败: ${volumeFilePath}`, err.message);
      return false;
    }
  }

  // 所有卷都通过检查
  console.log(`✅ 书籍《${book.name}》所有卷均已完整爬取`);
  return true;
}

const createBrowser = async () => {
  const launchOptions = {
    headless: true,
    executablePath: '/usr/bin/google-chrome',
    args: [
      // 🔧 稳定性 & 资源优化（优先级最高）
      '--no-sandbox',
      '--no-zygote',
      '--disable-setuid-sandbox',
      '--disable-dev-shm-usage',           // ⭐ 关键：避免 /dev/shm 不足
      '--disable-ipc-flooding-protection', // 减少 IPC 压力
      '--disable-background-networking',   // 禁用后台网络活动
      '--disable-background-timer-throttling',
      '--disable-renderer-backgrounding',
      '--disable-component-extensions-with-background-pages',
      '--disable-gpu',

      '--disable-web-security',
      '--allow-running-insecure-content',
      '--disable-site-isolation-trials',

      // 🚫 禁用网络服务（关键！减少 ERR_NETWORK_CHANGED）
      '--disable-features=NetworkService,TranslateUI,MediaRouter',
      '--disable-network-throttling',

      // 💾 缓存控制（可选，根据需求）
      '--disable-cache',
      '--disable-http-cache',
      '--disk-cache-size=0',
      '--media-cache-size=0',

      // 🖥 显示设置
      '--window-size=1920,1080',
      '--window-position=0,0',
      '--lang=zh-CN',
      '--accept-lang=zh-CN,zh',

      // 🔇 媒体与通知
      '--mute-audio',
      '--disable-notifications',
      '--use-fake-ui-for-media-stream',
      '--use-fake-device-for-media-stream',

      // 🛡 安全与自动化绕过（谨慎使用）
      '--disable-blink-features=AutomationControlled',
      '--ignore-certificate-errors',
      '--ignore-ssl-errors',

      // 🚫 其他无用功能
      '--no-first-run',
      '--no-default-browser-check',
      '--disable-infobars',
      '--disable-extensions',
      '--disable-default-apps',
      '--disable-translate',
      '--disable-sync',
      '--disable-client-side-phishing-detection',
      '--disable-component-update',
      '--disable-breakpad',
      '--disable-dev-shm-usage',
      '--disable-features=VizDisplayCompositor',
    ],
    defaultViewport: { width: 1920, height: 1080, deviceScaleFactor: 1 },
    ignoreDefaultArgs: [
      '--enable-automation',
      'enable-automation', // 字符串形式更保险
    ],
    protocolTimeout: 120000,
  };
  // 👇 在这里添加 User-Agent
  const userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.6261.95 Safari/537.36';
  launchOptions.args.push(`--user-agent=${userAgent}`);

  browser = await puppeteer.launch(launchOptions);
  browserPid = browser.process().pid;
  console.log(`🟢 启动浏览器，PID: ${browserPid}`);
  return browser;
};

/**
 * 安全跳转：在传入的 page 上跳转，失败则关闭并新建 page 重试
 * @param {Page} page - 要复用的页面实例（可为 null）
 * @param {string} url - 目标 URL
 * @returns {Promise<Page>} 成功加载的 page 实例（可能是新创建的）
 */
async function safeGoto(page, url) {
  const
    maxRetries = 3,
    timeout = 60000,
    waitUntil = ['domcontentloaded', 'networkidle0'],
    retryDelay = 1000;

  for (let i = 0; i <= maxRetries; i++) {
    // 如果不是第一次，或传入的 page 无效，则新建
    if (!page) {
      if (!browser) browser = await createBrowser();
      page = await browser.newPage();
      await page.setRequestInterception(true);
      page.on('request', (req) => {
        if (['image', 'stylesheet', 'font', 'media'].includes(req.resourceType())) {
          req.abort();
        } else {
          req.continue();
        }
      });
    }

    try {
      // 尝试跳转
      await page.goto(url, { waitUntil, timeout });
      console.log(`    ✅ 成功加载`);
      return page; // ✅ 成功：返回当前 page
    } catch (error) {
      await page.close();
      page = null; // 下次重试新建
      const isNetworkError = error.message.includes('net::ERR_NETWORK_CHANGED') ||
        error.message.includes('Navigation timeout') ||
        error.message.includes('net::ERR_CONNECTION') ||
        error.message.includes('net::ERR_ABORTED');

      // 非网络错误（如 404），不重试
      if (!isNetworkError) {
        console.log(`safeGoto 失败（非网络错误）: ${url} | ${error.message}`);
      }

      if (i < maxRetries) {
        const delay = retryDelay * Math.pow(2, i);
        console.warn(`⚠️ 跳转失败 (第 ${i + 1} 次): ${url} | ${error.message}`);
        console.log(`🔁 ${delay}ms 后重试...`);
        await sleep(delay);
      }
    }
  }

  // 所有重试失败
  await browser.close().catch(e => console.log('browser.close() 失败，忽略:', e.message));
  browser = null;
  await sleep(1000);

  if (browserPid) {
    try {
      // 先尝试优雅关闭
      process.kill(browserPid, 'SIGTERM');
      console.log(`🟡 发送 SIGTERM 给进程 ${browserPid}`);
    } catch (e) {
      console.log(`进程 ${browserPid} 已不存在 (SIGTERM)`);
    }
    // 等待 2 秒
    await sleep(2000);
    // 强制杀死
    try {
      process.kill(browserPid, 'SIGKILL');
      console.log(`🔴 强制杀死进程 ${browserPid}`);
    } catch (e) {
      console.log(`进程 ${browserPid} 已不存在 (SIGKILL)`);
    }

    browserPid = null;
    browser = null;
  }
  console.log(`🔁🔁🔁 重开浏览器...`);
  await sleep(3 * 1000)
  return await safeGoto(null, url);
}

(async () => {
  browser = await createBrowser();
  await browser.newPage();
  let page = await browser.newPage();
  // 减少负载：拦截图片、CSS 等
  await page.setRequestInterception(true);
  page.on('request', (req) => {
    if (['image', 'stylesheet', 'font', 'media'].includes(req.resourceType())) {
      req.abort();
    } else {
      req.continue();
    }
  });

  const docIndexPath = path.join(outputDir, `${baseSite}_site.json`);
  const docProgressPath = path.join(outputDir, `${baseSite}_progress.json`);
  let uniqueBooks;
  let booksProgress = {};
  if (fs.existsSync(docProgressPath)) {
    const data = fs.readFileSync(docProgressPath, 'utf8');
    booksProgress = JSON.parse(data);
  }

  try {
    if (fs.existsSync(docIndexPath)) {
      const data = fs.readFileSync(docIndexPath, 'utf8');
      uniqueBooks = JSON.parse(data);
      console.log(`✅ 成功找到 site.json，共 ${uniqueBooks.length} 本书`);
    } else {
      page = await safeGoto(page, 'https://www.zhiyuanzhongyi.com/medical');

      const tabs = await page.$$('div.medical_container div[role="tablist"] > div[role="tab"]');
      const allBooks = [];
      const booksByCategory = new Map();

      for (const tab of tabs) {
        const tabName = await tab.evaluate(el => el.textContent.trim());
        console.log(`✅ 点击分类: ${tabName}`);

        await tab.click();
        await sleep(1500);

        const books = await page.$$eval(
          `#pane-${tabName} .book_content_item`,
          (items, currentTabName) => {
            return items.map(item => {
              const link = item.querySelector('a');
              if (!link) return null;

              const href = link.href;
              const idMatch = href.match(/id=(\d+)/);
              if (!idMatch) return null;

              const id = parseInt(idMatch[1]);
              const name = item.querySelector('.book_name')?.textContent.trim() || '';
              const author = item.querySelector('.book_author span')?.textContent.trim() || '未知';
              const desc = item.querySelector('.book_desc p')?.textContent.trim() || '';

              if (!name) return null;

              return {
                id,
                name,
                author,
                desc,
                url: href,
                tags: [currentTabName]
              };
            }).filter(Boolean);
          },
          tabName
        );

        console.log(`📚 ${tabName} 提取到 ${books.length} 本书`);
        allBooks.push(...books);
      }

      // 去重
      const map = new Map();
      for (const book of allBooks) {
        if (map.has(book.id)) {
          if (!map.get(book.id).tags.includes(book.tags[0])) {
            map.get(book.id).tags.push(book.tags[0]);
          }
        } else {
          map.set(book.id, { ...book });
        }
        const category = book.tags[0];
        if (!booksByCategory.get(category)) {
          booksByCategory.set(category, []);
        }
        booksByCategory.get(category).push(book);
      }
      uniqueBooks = [...map.values()];

      // 保存 site.json
      fs.writeFileSync(docIndexPath, JSON.stringify(uniqueBooks, null, 2), 'utf8');
      console.log(`✅ 成功生成 site.json，共 ${uniqueBooks.length} 本书`);

      // 生成总目录
      let siteIndexContent = `# ${baseSite} 书籍总目录\n\n> 共收录 ${uniqueBooks.length} 本书\n\n`;
      for (const [category, books] of booksByCategory) {
        siteIndexContent += `## ${category}\n\n`;
        books.forEach(book => {
          const bookDirName = encodeURIComponent(book.name.replace(/[\\/:\*\?\"<>\|]/g, ''));
          siteIndexContent += `- [${book.name}](${bookDirName}/)（${book.author}）\n`;
          if (book.desc) {
            siteIndexContent += `  > ${book.desc}\n`;
          }
          siteIndexContent += '\n';
        });
        siteIndexContent += '\n';
      }

      const siteIndexFilename = path.join(outputDir, `${baseSite}_目录.md`);
      fs.writeFileSync(siteIndexFilename, siteIndexContent, 'utf8');
      console.log(`📄 网站总目录已生成: ${siteIndexFilename}`);
    }

    // 遍历每本书
    let progressIndex = 0;
    for (const book of uniqueBooks) {
      const { id, url } = book;
      const bookDir = path.join(outputDir, book.name);
      if (!allowBooks.includes(id)) {
        if (fs.existsSync(bookDir)) {
          console.log(`🔍 正在删除书籍`);
          deleteFolderRecursive(bookDir)
        }
        console.log(`⏭️ 跳过非核心书籍: ${book.name}`);
        continue;
      }

      if (!fs.existsSync(bookDir)) fs.mkdirSync(bookDir, { recursive: true });
      progressIndex++;

      // ✅ 增量：跳过已爬取的书
      if (fs.existsSync(path.join(bookDir, '000_目录.md')) && booksProgress[book.name]) {
        console.log(`⏭️ 跳过已爬取书籍: ${book.name}    ${progressIndex}/81`);
        continue;
      }

      console.log(`🔍 正在爬取书籍: ${book.name} (ID=${id})    ${progressIndex}/81`);

      page = await safeGoto(page, url);
      let bookTitle = await page.evaluate(() => {
        return document.querySelector('.book_title')?.innerText.trim();
      });
      if (!bookTitle) {
        console.log(`🔍 正在爬取书籍++`);
        page = await safeGoto(page, url);
        bookTitle = await page.evaluate(() => {
          return document.querySelector('.book_title')?.innerText.trim();
        });
        if (!bookTitle) {
          console.log(` 获取《${book.name}》 未知书名 跳过 |`, url);
          continue;
        }
      }

      // ✅ 获取书籍结构数据（支持“无标题卷”使用章节名作为卷名）
      const bookData = await page.evaluate((bookId) => {
        const bookTitle = document.querySelector('.book_title')?.innerText.trim() || '未知书名';
        const volumeElements = document.querySelectorAll('.catalog_item');
        const volumes = [];

        volumeElements.forEach((volEl, index) => {
          const volumeTitleEl = volEl.querySelector('.catalog_title');
          const volumeTitle = volumeTitleEl ? volumeTitleEl.innerText.trim() : '';
          const chapterEls = volEl.querySelectorAll('.catalog_chapter a');

          const chapters = [];
          chapterEls.forEach(a => {
            const chapterName = a.querySelector('.chapter_name')?.innerText.trim() || '未知章节';
            const href = a.getAttribute('href');
            const fullUrl = new URL(href, location.origin).href;
            chapters.push({ chapterName, url: fullUrl });
          });

          if (volumeTitle) {
            // 有标题 → 正常卷
            volumes.push({
              type: 'normal',
              index,
              volumeTitle,
              chapters
            });
          } else if (chapters.length > 0) {
            // 无标题但有章节 → 视为独立章节卷
            volumes.push({
              type: 'chapter-as-volume',
              index,
              volumeTitle: chapters[0].chapterName, // 用第一个章节名作为卷名
              chapterUrl: chapters[0].url,
              chapters: []
            });
          } else {
            // 空卷（极少）
            volumes.push({
              type: 'empty',
              index,
              volumeTitle: `空卷-${index + 1}`,
              chapters: []
            });
          }
        });

        return { id: bookId, bookTitle, volumes };
      }, id);

      console.log(`✅ 获取《${bookData.bookTitle}》共 ${bookData.volumes.length} 卷`);

      // ✅ 生成 000_目录.md —— 清晰展示“卷+章节”结构，无“未命名卷”
      let docMd = `# ${bookData.bookTitle}\n\n`;
      docMd += `> 《[${baseSite}-${bookData.bookTitle}（${book.author}）](${url})》 书籍 ID: ${bookData.id}\n\n`;

      for (const vol of bookData.volumes) {
        const safeVolumeTitle = vol.volumeTitle.replace(/[\\/:\*\?\"<>\|]/g, '_');
        const volumeFilename = `${String(vol.index + 1).padStart(3, '0')}_${safeVolumeTitle}.md`;
        const volumeUrlFilename = encodeURIComponent(volumeFilename);

        docMd += `## ${vol.volumeTitle}\n\n`;
        docMd += `- [📖 ${vol.volumeTitle}](${volumeUrlFilename})\n\n`;

        if (vol.type === 'normal' && vol.chapters.length > 0) {
          docMd += `  > 本卷包含 ${vol.chapters.length} 个章节：\n`;
          vol.chapters.forEach(ch => {
            docMd += `  - ${ch.chapterName}\n`;
          });
          docMd += '\n';
        } else if (vol.type === 'chapter-as-volume') {
          docMd += `  > （本卷对应独立章节，已合并内容）\n\n`;
        } else {
          docMd += `  > （本卷暂无内容）\n\n`;
        }
      }

      const docPath = path.join(bookDir, '000_目录.md');
      fs.writeFileSync(docPath, docMd, 'utf8');
      console.log(`📄 目录已保存: ${docPath}`);

      // ✅ 遍历每一卷并生成文件
      const failedKeywords = ['爬取失败：', '内容未获取到。', '本卷为空，无内容。'];
      for (let i = 0; i < bookData.volumes.length; i++) {
        const vol = bookData.volumes[i];
        const safeVolumeTitle = vol.volumeTitle.replace(/[\\/:\*\?\"<>\|]/g, '_');
        const volumeFilename = `${String(i + 1).padStart(3, '0')}_${safeVolumeTitle}.md`;
        const volumeFilePath = path.join(bookDir, volumeFilename);
        // ✅ 断点续爬判断：文件存在 且 内容不含“爬取失败” 才视为已完成
        if (fs.existsSync(volumeFilePath)) {
          const content = fs.readFileSync(volumeFilePath, 'utf8');
          let ok = true
          if (failedKeywords.some(keyword => content.includes(keyword))) {
            console.log(`🔍 文件中包含失败标记: ${volumeFilename}`);
            ok = false;
          }
          if (ok) {
            console.log(`⏭️ 跳过已完整爬取的卷: ${volumeFilename}`);
            continue;
          } else {
            console.log(`🔄 重新爬取部分失败的卷: ${volumeFilename}`);
          }
        }
        console.log(`⏭️ 开始处理卷: ${String(i + 1).padStart(3, '0')}_${safeVolumeTitle} / ${bookData.volumes.length}卷-${bookData.bookTitle}`);
        let volumeMd = `# ${bookData.bookTitle} · ${vol.volumeTitle}\n\n`;

        if (vol.type === 'normal') {
          // 正常卷：合并所有章节
          let ch_index = 0
          for (const ch of vol.chapters) {
            const paddedChapterName = ch.chapterName.padEnd(10, '　'); // 使用中文全角空格补足到10个字符
            console.log(`    📖 正在爬取章节: ${paddedChapterName} -- ${String(ch_index + 1).padStart(3)}/${vol.chapters.length}章进度  ${String(i + 1).padStart(2)}/${bookData.volumes.length}卷进度    ${progressIndex}/81${bookData.bookTitle}`);
            try {
              page = await safeGoto(page, ch.url);
              const contentHtml = await page.evaluate(() => {
                const el = document.querySelector('.chapter_content');
                return el ? el.innerHTML : null;
              });

              if (contentHtml === null) {
                volumeMd += `### ${ch.chapterName}\n\n> 内容未获取到。\n\n`;
                continue;
              }

              let markdown = turndownService.turndown(contentHtml);
              markdown = markdown.replace(/\n{3,}/g, '\n\n');
              if (!markdown) {
                markdown = '<空>'
              }

              volumeMd += `### ${ch.chapterName}\n\n`;
              volumeMd += markdown + '\n\n';
              console.log(`    ✅ 已添加: ${ch.chapterName}`);
            } catch (err) {
              console.error(`    ❌ 爬取章节失败: ${ch.chapterName}`, err.message);
              volumeMd += `### ${ch.chapterName}\n\n> 爬取失败：${err.message}\n\n`;
            }
            ch_index++;
          }
        } else if (vol.type === 'chapter-as-volume') {
          // 独立章节卷：直接爬取该章节
          try {
            console.log(`    📖 正在爬取独立章节: ${vol.volumeTitle}`);
            page = await safeGoto(page, vol.chapterUrl);
            const contentHtml = await page.evaluate(() => {
              const el = document.querySelector('.chapter_content');
              return el ? el.innerHTML : null;
            });

            if (contentHtml === null) {
              volumeMd += `> 内容未获取到。\n\n`;
            } else if (!contentHtml) {
              volumeMd += `> <空>\n\n`;
            } else {
              let markdown = turndownService.turndown(contentHtml);
              markdown = markdown.replace(/\n{3,}/g, '\n\n');
              volumeMd += markdown + '\n\n';
            }
            console.log(`    ✅ 已添加: ${vol.volumeTitle}`);
          } catch (err) {
            console.error(`    ❌ 爬取独立章节失败: ${vol.volumeTitle}`, err.message);
            volumeMd += `> 爬取失败：${err.message}\n\n`;
          }
        } else {
          volumeMd += `> 本卷为空，无内容。\n\n`;
        }

        fs.writeFileSync(volumeFilePath, volumeMd, 'utf8');
        console.log(`    💾 已保存卷文件: ${volumeFilename}`);
      }

      // ✅ 校验书籍是否完成
      const ok = isBookComplete(bookDir, book, bookData.volumes);
      if (ok) {
        booksProgress[book.name] = true;
        fs.writeFileSync(docProgressPath, JSON.stringify(booksProgress, null, 2), 'utf8');
      } else {
        console.log(`    💾 ${book.name} 保存卷文件不全`);
        await sleep(3000)
      }
    }

    console.log('🎉 所有书籍爬取完成！');
  } catch (err) {
    console.error('❌ 主流程出错:', err);
  } finally {
    await browser.close();
  }
})();